package cmm;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.*;

import clusterMessages.ClusterMessage;
import clusterMessages.ClusterMessage_AppMigrate;
import clusterMessages.ClusterMessage_AppNewTask;
import clusterMessages.ClusterMessage_AppTerminateTask;
import clusterMessages.ClusterMessage_AppTransferTask;
import clusterMessages.ClusterMessage_AppUpdateTask;
import clusterMessages.ClusterMessage_NewServerReady;
import clusterMessages.ClusterMessage_ProxyMigrate;
import clusterMessages.ClusterMessage_ProxyNewTask;
import clusterMessages.ClusterMessage_RemoveMachine;
import clusterMessages.ClusterMessage_TrainingMigrate;
import clusterMessages.ClusterMessage_TrainingNewTask;
import clusterMessages.ClusterMessage_TrainingTerminateTask;
import clusterMessages.ClusterMessage_TrainingUpdateTask;
import clusterMessages.MachineTrainingData;
import clusterMessages.TaskTrainingData;

import server.SAGenTaskContext;
import server.ServerApplication;

import spread.*;

public class ClusterManagementModule implements IClusterManagementModule {
	
	private TreeMap<Integer, TaskEntry> TaskDB;
	
	/**
	 * Holds the number of tasks in the whole Cluster.
	 */
	private int ClusterTaskCount = 0;
	
	// This is the Fault Tolerance and Load Balancing "Watermark" 
	
	/**
	 * Holds the number of the ACTIVE machines in the Cluster.
	 * This is used for the ClusterTaskAverage calculation.
	 */
	private int ClusterActiveMachineCount = 0;
	
	private TreeMap<Integer, MachineEntry> MachineDB;
	
	private ProxyServer Proxy;
	
	private Integer ProxyMachineID = null;
	
	private String ProxyMask;
	
	private String DefaultGatewayIP;
	
	private SpreadConnection ClusterConnection;
	
	private SpreadGroup ClusterGroup;
	
	private String GroupName;
	
	private Integer MachineDBLocalID;
	
	private ServerApplication SA;
	
	private AdvancedSpreadListener SpreadListener;
	
	private LinkedList<ToBeMentoredEntry> ToBeMentoredList;
	
	private LinkedList<Mentor> MentorList;
	
	private MachineEntry LocalMachineEntry;
	
	private MessageQueue TrainingMessageQueue;
	
	private boolean InteractiveTrainingMode = false;
	
	private String NetworkConnectionName;
	
//------------------------------------------------------------------------
//  Private Methods	
//------------------------------------------------------------------------
	
	/**
	 * The method sends messages to the whole cluster via the 
	 * ClusterConnection. All cluster communication should be done 
	 * through this method. 
	 * 
	 * @param ClusterMSG
	 * @throws CMMExn_ClusterErr 
	 */
	private void SendClusterMessage(ClusterMessage ClusterMSG) throws CMMExn_ClusterErr{
		
		SA.AddLogEntry("CMM", "Info - Sending a message to the cluster. Message Type - " + ClusterMSG.getType());
		
		// Set this machine to be the sender machine 
		ClusterMSG.setSenderID(MachineDBLocalID);
		
		try {
			// Create a new SpreadMessage
			SpreadMessage Message = new SpreadMessage();
			Message.setObject(ClusterMSG);
			Message.addGroup(ClusterGroup);
			Message.setAgreed();
			
			// Send the message to the cluster
			ClusterConnection.multicast(Message);
		} catch (SpreadException e) {
			// Report failure to send ClusterMessage
			e.printStackTrace();
			SA.AddLogEntry("CMM", "Error - Failed to send a message to the cluster." + 
					" MessageType : " + ClusterMSG.getType());
			
			// Throw a Cluster Exception
			CMMExn_ClusterErr exn = new CMMExn_ClusterErr();
			throw exn;
		}
	}

//	------------------------------------------------------------------------	
	
	private void ProxyNewTask(ConnectionInformation ProxyConnInfo) throws CMMExn_ClusterErr {

		SA.AddLogEntry("CMM", "Info - Sending a \"Proxy New Task\" request to the Cluster.");
		
		// Create a Proxy Context to send
		ProxyContext Context = new ProxyContext();
		Context.putConnInfo(ProxyConnInfo);
		
		// Create a "New Task" message 
		ClusterMessage_ProxyNewTask msg = new ClusterMessage_ProxyNewTask(); 
		msg.setProxyContext(Context);
		
		// Send the message through the cluster
		try {
			SendClusterMessage(msg);
		} catch (CMMExn_ClusterErr e) {
			SA.AddLogEntry("CMM", "Error - Failed to send \"New Task\" message to the cluster.");
			throw e;
		}
	}
	
//	------------------------------------------------------------------------	
	
	/**
	 * This method inserts a {@link TaskContext} as a new {@link TaskEntry}
	 * into the Task Database
	 * 
	 * @param MachineID - The MachineID running the Task
	 * @param Context - The {@link TaskContext}
	 * @return The new TaskID of the new Task 
	 * @throws CMMExn_TaskErr - MachineID does not exist in the Machine Database
	 */
	private synchronized Integer InsertNewTaskIntoDB(Integer MachineID, TaskContext Context) throws CMMExn_TaskErr{
		
		// Verify that the machine ID exists in the machine database
		if(!MachineDB.containsKey(MachineID))
		{
			SA.AddLogEntry("CMM", "Error - MachineID " + MachineID + " does not exist in the Machine Database.");
			CMMExn_TaskErr exn = new CMMExn_TaskErr();
			throw exn;
		}
	
		// Retrieve previous maximum task ID
		Integer MaxTaskID;
		if(TaskDB.isEmpty())
			MaxTaskID = new Integer(-1);
		else
			MaxTaskID = TaskDB.lastKey();
	
		// Create a new task ID
		// New task ID is prev task ID incramented by 1
		Integer NewTaskID = new Integer(MaxTaskID.intValue()+1);
	
		// Create a new task entry
		TaskEntry NewEntry = new TaskEntry(MachineID, Context);
	
		// Add the new task entry to the task database
		TaskDB.put(NewTaskID, NewEntry);
		
		// Update the task amount entry of the relevant machine in
		// the machine database
		// Retrieve prev machine entry
		MachineEntry MEntry = MachineDB.get(MachineID);
		// Incrament the task count 
		MEntry.incTaskCount();
		
		ClusterTaskCount++;
		
		return NewTaskID;
	}

//------------------------------------------------------------------------

	/**
	 * This method processes the Task queue that was filled up, while server was
	 * in training. It is also called at Mentor Migration, and then it processes
	 * only the Tasks up to the set maximum.
	 * 
	 * @param MaxTaskID - The maximum TaskID to be processed. If null, then all tasks
	 * will be processed. Maximum should be set only in Mentor Migration
	 */
	private synchronized void ProcessTaskQueue(Integer MaxTaskID) {

		while(!TrainingMessageQueue.isEmpty()){
			ClusterMessage ClusterMSG = TrainingMessageQueue.Pop();
			switch(ClusterMSG.getType()){
			// Application Messages
				case APP_NEW_TASK :
				{
					// if there's a TaskID maximum
					if(MaxTaskID != null)
						// ignore new tasks
						break;
					// Convert the message to a "Application New Task" message object
					ClusterMessage_AppNewTask msg = (ClusterMessage_AppNewTask)ClusterMSG;
					// Inform the CMM of a "Application New Task" message
					ReceiveAppNewTask(msg.getSenderID(), msg.getTaskContext());
					break;
				}
				case APP_UPDATE_TASK :
				{	
					// Convert the message to a "Application Update Task" message object
					ClusterMessage_AppUpdateTask msg = (ClusterMessage_AppUpdateTask)ClusterMSG;

					// if there's a TaskID maximum, and this message involves TaskID above this maximum.
					if((MaxTaskID != null) && (msg.getTaskID().intValue() > MaxTaskID.intValue()))
							break;
					
					// Inform the CMM of an "Application Update Task" message
					ReceiveAppUpdateTask(msg.getSenderID(), msg.getTaskID(), msg.getContextUpdate());
					break;
				}
				case APP_TERMINATE_TASK :
				{
					// Convert the message to a "Application Terminate Task" message object
					ClusterMessage_AppTerminateTask msg = (ClusterMessage_AppTerminateTask)ClusterMSG;
					
					// if there's a TaskID maximum, and this message involves TaskID above this maximum.
					if((MaxTaskID != null) && (msg.getTaskID().intValue() > MaxTaskID.intValue()))
							break;
					
					// Inform the CMM of an "Application Terminate Task" message
					ReceiveAppTerminateTask(msg.getSenderID(), msg.getTaskID());
					break;
				}
				case APP_MIGRATE_TASK :
				{
					// Convert the message to a "Application Migrate Task" message object
					ClusterMessage_AppMigrate msg = (ClusterMessage_AppMigrate)ClusterMSG;
					
					// if there's a TaskID maximum, and this message involves TaskID above this maximum.
					if((MaxTaskID != null) && (msg.getTaskID().intValue() > MaxTaskID.intValue()))
							break;
					
					// Inform the CMM of an "Application Migrate Task" message
					ReceiveAppMigrate(msg.getSenderID(), msg.getTaskID());
					break;
				}
				case APP_TRANSFER_TASK :
				{
					// Convert the message to a "Application Transfer Task" message object
					ClusterMessage_AppTransferTask msg = (ClusterMessage_AppTransferTask)ClusterMSG;
					
					// if there's a TaskID maximum, and this message involves TaskID above this maximum.
					if((MaxTaskID != null) && (msg.getTaskID().intValue() > MaxTaskID.intValue()))
							break;
					
					// Inform the CMM of an "Application Transfer Task" message
					ReceiveAppTransferTask(msg.getSenderID(), msg.getTaskID(), msg.getNewOwnerMachineID());
					break;
				}
			// Proxy Messages
				case PROXY_NEW_TASK :
				{
					// Convert the message to a "Proxy New Task" message object
					ClusterMessage_ProxyNewTask msg = (ClusterMessage_ProxyNewTask)ClusterMSG;
					
					// if there's a TaskID maximum
					if(MaxTaskID != null)
							break;
					
					// Inform the CMM of a "Proxy New Task" message
					ReceiveProxyNewTask(msg.getSenderID(), msg.getProxyContext());
					break;
				}
				case PROXY_MIGRATE_TASK :
				{
					// Convert the message to a "Proxy Migrate Task" message object
					ClusterMessage_ProxyMigrate msg = (ClusterMessage_ProxyMigrate)ClusterMSG;
					
					// if there's a TaskID maximum, and this message involves TaskID above this maximum.
					if((MaxTaskID != null) && (msg.getTaskID().intValue() > MaxTaskID.intValue()))
							break;
					
					// Inform the CMM of an "Proxy Migrate Task" message
					ReceiveProxyMigrate(msg.getSenderID(), msg.getTaskID());
					break;
				}
			// CMM Messages
				// note: this is not a real message and it is not recieved from the cluster.
				// it may only come from the Training Message Queue.
				case REMOVE_MACHINE :
				{
					// Convert the message to a "Remove Machine" message object
					ClusterMessage_RemoveMachine msg = (ClusterMessage_RemoveMachine)ClusterMSG;
					// Remove the leaving machine from the Machine Database
					RemoveMachineFromCluster(msg.getSenderID());
					break;
					
				}
				case NEW_SERVER_READY :
				{
					// Convert the message to a "New Server Ready" message object
					ClusterMessage_NewServerReady msg = (ClusterMessage_NewServerReady)ClusterMSG;
					// Inform the CMM of a "New Server Ready" message
					ReceiveNewServerReady(msg.getNewMachineEntry());
					break;
				}
				case TRAINING_NEW_TASK :
				{
					// if there's a TaskID maximum
					if(MaxTaskID != null)
						// ignore
						break;
					
					// Convert the message to a "Training New Task" message object
					ClusterMessage_TrainingNewTask msg = (ClusterMessage_TrainingNewTask)ClusterMSG;
					// Inform the CMM of a "Training New Task" message
					ReceiveTrainingNewTask(msg.getSenderID(), msg.getData());
					break;
				}
				case TRAINING_UPDATE_TASK :
				{	
					// Convert the message to a "Training Update Task" message object
					ClusterMessage_TrainingUpdateTask msg = (ClusterMessage_TrainingUpdateTask)ClusterMSG;

					// if there's a TaskID maximum, and this message involves TaskID above this maximum.					
					if((MaxTaskID != null) && (msg.getTaskID() > MaxTaskID))
						// ignore
						break;
					
					// Inform the CMM of a "Training Update Task" message
					ReceiveTrainingUpdateTask(msg.getSenderID(), msg.getTaskID(), msg.getData(), msg);
					break;
				}
				case TRAINING_TERMINATE_TASK :
				{
					// Convert the message to a "Training Terminate Task" message object
					ClusterMessage_TrainingTerminateTask msg = (ClusterMessage_TrainingTerminateTask)ClusterMSG;
					
					// if there's a TaskID maximum, and this message involves TaskID above this maximum.					
					if((MaxTaskID != null) && (msg.getTaskID() > MaxTaskID))
						// ignore
						break;
					
					// Inform the CMM of a "Training Terminate Task" message
					ReceiveTrainingTerminateTask(msg.getSenderID(), msg.getTaskID(), msg.getTraineeName(), msg);
					break;
				}
				case TRAINING_MIGRATE_TASK :
				{
					// Convert the message to a "Training Migrate Task" message object
					ClusterMessage_TrainingMigrate msg = (ClusterMessage_TrainingMigrate)ClusterMSG;
					
					// if there's a TaskID maximum, and this message involves TaskID above this maximum.					
					if((MaxTaskID != null) && (msg.getTaskID() > MaxTaskID))
						// ignore
						break;
					
					// Inform the CMM of a "Training Migrate Task" message
					ReceiveTrainingMigrate(msg.getSenderID(), msg.getTaskID(), msg.getTraineeName(), msg);
					break;
				}	
				default :
					// Inform the CMM that an unsupported message was recieved
					ReceiveUnsupportedMessage();
			}
		}
	}

//	------------------------------------------------------------------------	
	
	private void WaitForEnter(String MessageToUser) {

			System.out.println(MessageToUser);
			BufferedReader userIn =
		        new BufferedReader(
		        new InputStreamReader( System.in ) );
			
			try {
				userIn.readLine();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
		}
	}
	
//	------------------------------------------------------------------------	
	
	private synchronized void RemoveMachineFromCluster(Integer MachineID) {
		
		MachineDB.remove(MachineID);
		SA.AddLogEntry("CMM", "Info - Removed Machine with MachineID " + MachineID + " from the Machine Database.");
		
		// An ACTIVE machine has left the Cluster, so dec the count.
		ClusterActiveMachineCount--;
	}

//	------------------------------------------------------------------------	
	
	/**
	 * This method is called every time a machine leaves the Cluster. 
	 * The method triggers the fault tolerance algorythm.
	 * 
	 * @param LeaverMachineID - The MachineID of the leaver machine
	 */
	private synchronized void ExecuteFaultTolerance(Integer LeaverMachineID) {
		
		SA.AddLogEntry("CMM", "Info - Executing Fault Tolerance algorythm because Server with MachineID=" + LeaverMachineID + " left the Cluster");
		
		int taskWaterMark = TaskWaterMark();
		
		SA.AddLogEntry("CMM", "Info - New Task Watermark is " + taskWaterMark);
		
		// Find all the tasks of the leaving machine, and put them in their own database.
		// Create a new Task Database only for the UnAllocated Tasks
		TreeMap<Integer, TaskEntry> UnAllocatedTaskDB = new TreeMap<Integer, TaskEntry>();
		// Convert TaskDB from TreeMap into a Set
		Set<Map.Entry<Integer, TaskEntry>> TaskSet = TaskDB.entrySet();
		Iterator<Map.Entry<Integer, TaskEntry>> Tit = TaskSet.iterator();
		while(Tit.hasNext()){
			Map.Entry<Integer, TaskEntry> TMapEntry = Tit.next();
			// If this task is running on the leaving server
			if(TMapEntry.getValue().getAssignedServerID().equals(LeaverMachineID))
				// Nullify the Assigned ServerID entry of the Task
				TMapEntry.getValue().putAssignedServerID(null);
			if(TMapEntry.getValue().getAssignedServerID() == null){
				// put the task entry in the UnAllocated Tasks Database
				UnAllocatedTaskDB.put(TMapEntry.getKey(), TMapEntry.getValue());
			}
		}
		
		// Calculate which tasks should be taken by this machine
		int StartingTaskNum = 0;
		int NumOfTasksToTake = 0;
		// Convert MachineDB from TreeMap into a Set
		Set<Map.Entry<Integer, MachineEntry>> MachineSet = MachineDB.entrySet(); 
		Iterator<Map.Entry<Integer, MachineEntry>> it = MachineSet.iterator();
		boolean FoundLocalMachine = false;
		// for each element in the set
		while(it.hasNext() && !FoundLocalMachine){
			// retrieve the set element
			Map.Entry<Integer, MachineEntry> MapEntry = it.next();
			// If this entry is not of this local machine
			if(!MapEntry.getKey().equals(MachineDBLocalID)){
				// If this entry's machine's Task count is not above the WaterMark
				if(taskWaterMark >= MapEntry.getValue().getTaskCount())
					// The local machine will start taking the tasks that
					// are after the tasks that this machine will take.
					StartingTaskNum += taskWaterMark - MapEntry.getValue().getTaskCount();
			}
			// if this entry is this local machine
			else{
				if(taskWaterMark >= LocalMachineEntry.getTaskCount())
					NumOfTasksToTake = taskWaterMark - LocalMachineEntry.getTaskCount();
				FoundLocalMachine = true;
			}
		}
		
		SA.AddLogEntry("CMM", "Info - Going to take ownership of " + NumOfTasksToTake + " tasks.");
		
		// Take ownership of the appropriate unallocated Tasks
		int TaskCounter = -1;
		TaskSet = UnAllocatedTaskDB.entrySet();
		Iterator<Map.Entry<Integer, TaskEntry>> TaskIterator = TaskSet.iterator();
		while(TaskIterator.hasNext()){
			TaskCounter++;
			Map.Entry<Integer, TaskEntry> TaskMapEntry = TaskIterator.next();
			
			// if didn't get to the first task to be taken over  
			if(TaskCounter < StartingTaskNum)
				continue;
			
			// if past over all the tasks to be taken over
			if(TaskCounter >= StartingTaskNum + NumOfTasksToTake)
				break;
			
			SA.AddLogEntry("CMM", "Info - Taking ownership of task with TaskID=" + TaskMapEntry.getKey());
			
			// if the taken over task is of APPLICATION type
			if(TaskMapEntry.getValue().getContext().getType() == TaskType.APPLICATION){
				// Create an "Application Migration" message
				ClusterMessage_AppMigrate msg = new ClusterMessage_AppMigrate();
				// set the TaskID entry of the message to the current Task
				msg.setTaskID(TaskMapEntry.getKey());
				try {
					// Send the message to the Cluster
					SendClusterMessage(msg);
				} catch (CMMExn_ClusterErr e) {
					// TODO remove StackPrint 
					e.printStackTrace();
					SA.AddLogEntry("CMM", "Error - Failed to send \"Application Migration Task\" message to the Cluster.");
					SA.UnrecoverableClusterError();
				}
			}
			else if(TaskMapEntry.getValue().getContext().getType() == TaskType.PROXY){
				// Create an "Proxy Migration" message
				ClusterMessage_ProxyMigrate msg = new ClusterMessage_ProxyMigrate();
				// set the TaskID entry of the message to the current Task
				msg.setTaskID(TaskMapEntry.getKey());
				try {
					// Send the message to the Cluster
					SendClusterMessage(msg);
				} catch (CMMExn_ClusterErr e) {
					// TODO remove StackPrint 
					e.printStackTrace();
					SA.AddLogEntry("CMM", "Error - Failed to send \"Proxy Migration Task\" message to the Cluster.");
					SA.UnrecoverableClusterError();
				}
			}
			else if(TaskMapEntry.getValue().getContext().getType() == TaskType.TRAINING){
				// Create an "Training Migration" message
				ClusterMessage_TrainingMigrate msg = new ClusterMessage_TrainingMigrate();
				// set the TaskID entry of the message to the current Task
				msg.setTaskID(TaskMapEntry.getKey());
				// set the Trainee name entry of the message
				String TraineeName = ((TrainingContext)(TaskMapEntry.getValue().getContext())).getTraineeName();
				msg.setTraineeName(TraineeName);
				
				if(InteractiveTrainingMode)
					WaitForEnter("Press Enter to send \"Training Migrate Task\" message to " + TraineeName);
				
				try {
					// Send the message to the Cluster
					SendClusterMessage(msg);
				} catch (CMMExn_ClusterErr e) {
					// TODO remove StackPrint 
					e.printStackTrace();
					SA.AddLogEntry("CMM", "Error - Failed to send \"Training Migration Task\" message to the Cluster.");
					SA.UnrecoverableClusterError();
				}
			}
			else {
				SA.UnrecoverableClusterError();
			}
		}
	}

//	------------------------------------------------------------------------
	
private int TaskWaterMark() {	
	if(ClusterTaskCount % ClusterActiveMachineCount != 0)
		return ClusterTaskCount / ClusterActiveMachineCount + 1;
	else
		return ClusterTaskCount / ClusterActiveMachineCount;
}

//	------------------------------------------------------------------------
	
	/**
	 * This method goes over the TaskDB and MachineDB, 
	 * searches for Machines that have a Tasks count above the 
	 * Task-Watermark, decides which APPLICATION Tasks should be taken
	 * over from the other Machines to balance the Cluster 
	 * Task managing load.
	 * Finally, the method sends the appropriate "Migration" messages
	 * to the Cluster.
	 * 
	 *  note: This method is not synchronus with all the other Machines
	 *  in the Cluster. This means that when a single Machine executes 
	 *  this method, the other Machines are not aware of this 
	 *  (until they get the "Migration" messages).
	 *  Thus this method does not assure true load balancing and
	 *  should only be used once after a final joining to the Cluster
	 *  (after "New Server Ready" message). 
	 */
	private void ExecuteLoadBalancing(Integer NewMachineID) {
		
		SA.AddLogEntry("CMM", "Info - Executing Load Balancing algorythm.");
		
		int taskWaterMark = TaskWaterMark();
				
		SA.AddLogEntry("CMM", "Info - New Task Watermark is " + taskWaterMark);
		
		// Check if Task count in local machine is above the watermark. 
		if(LocalMachineEntry.getTaskCount() <= taskWaterMark){
			SA.AddLogEntry("CMM", "Info - Current Task count on local machine is not above the Task Watermark. " +
					"Local Machine won't transfer any Tasks to the new joined Machine");
			return;
		}
		
		int TasksToTransfer = LocalMachineEntry.getTaskCount() - taskWaterMark;
		
		SA.AddLogEntry("CMM", "Info - Current Task count on local machine is " +
				TasksToTransfer + " above the Task Watermark.");
		
		// Convert the TaskDB from a TreeMap into a Set
		Set<Map.Entry<Integer, TaskEntry>> TaskSet = TaskDB.entrySet();
		Iterator<Map.Entry<Integer, TaskEntry>> tIter = TaskSet.iterator();
		// Go over all the Tasks in the TaskDB
		while(tIter.hasNext()){
			Integer TaskID = null;
			synchronized(this){
				// retrieve the set element
				Map.Entry<Integer, TaskEntry> tMapEntry = tIter.next();
				// If this task is not an APPLICATION type
				if(tMapEntry.getValue().getContext().getType() != TaskType.APPLICATION)
					// skip it
					continue;

				// if Task is not assigned to this local Machine
				if(!tMapEntry.getValue().getAssignedServerID().equals(MachineDBLocalID)){
					// skip it
					continue;
				}
				
				// if got here: Task is an APPLICATION type assigned to this Machine
				SA.AddLogEntry("CMM", "Info - Will transfer Application Task " + 
						tMapEntry.getKey() + " to new Machine " + NewMachineID);
				TaskID = tMapEntry.getKey();
			}
			
			SA.AddLogEntry("CMM", "Info - Notifying the Server Application.");
			// This mustn't be synchronized, or will cause a dead-lock
			SA.StopServicingTask(TaskID);
			
			// Create an Aplication Transfer Message
			ClusterMessage_AppTransferTask msg = new ClusterMessage_AppTransferTask();
			msg.setSenderID(MachineDBLocalID);
			msg.setTaskID(TaskID);
			msg.setNewOwnerMachineID(NewMachineID);
			
			try {
				// Send the message to the Cluster
				SendClusterMessage(msg);
			} catch (CMMExn_ClusterErr e) {
				// TODO remove StackPrint 
				e.printStackTrace();
				SA.AddLogEntry("CMM", "Error - Failed to send \"Application Migration Task\" message to the Cluster.");
				SA.UnrecoverableClusterError();
			}
			
			TasksToTransfer--;
			// If transfered enough Tasks to get bellow the new Task Watermark
			if(TasksToTransfer == 0)
				break;
		}
		
		SA.AddLogEntry("CMM", "Info - Finished Load Balancing algorythm.");
	}
			
/*	
		// Create a database to remember how many tasks to take from each Machine
		// note: Key is MachineID
		TreeMap<Integer,Integer> TaskCountToBeTaken = new TreeMap<Integer, Integer>();
		// Convert MachineDB from TreeMap into a Set
		Set<Map.Entry<Integer, MachineEntry>> MachineSet = MachineDB.entrySet(); 
		Iterator<Map.Entry<Integer, MachineEntry>> mIter = MachineSet.iterator();
		// Go over all the Machines in the MachineDB
		while(mIter.hasNext()) {
			// retrieve the set element
			Map.Entry<Integer, MachineEntry> mMapEntry = mIter.next();
			// skip local machine
			if(mMapEntry.getKey().equals(MachineDBLocalID))
				continue;
			MachineEntry mEntry = mMapEntry.getValue();
			// Calculate how many tasks are above the Watermark in this Machine
			int TasksOverLimit = mEntry.getTaskCount() - taskWaterMark;
			if(TasksOverLimit > 0){
				// Remember to take TasksOverLimit Tasks from this Machine
				TaskCountToBeTaken.put(mMapEntry.getKey(), TasksOverLimit);
				SA.AddLogEntry("CMM", "Info - Task count of MachineID " + mMapEntry.getKey() + 
						" is over the Watermark. Local Machine will take " + TasksOverLimit +
						" tasks from it.");
			}
			else{
				// do not take any tasks from this Machine
				TaskCountToBeTaken.put(mMapEntry.getKey(), 0);
				SA.AddLogEntry("CMM", "Info - Task count of MachineID " + mMapEntry.getKey() +
						" is not above the Watermark. Local Machine will not take any tasks from it.");
			}
		}
		
		// Create a temp TaskDB of tasks to taken from other Machines
		TreeMap<Integer, TaskEntry> TasksToBeTaken = new TreeMap<Integer, TaskEntry>();
		// Convert the TaskDB from a TreeMap into a Set
		Set<Map.Entry<Integer, TaskEntry>> TaskSet = TaskDB.entrySet();
		Iterator<Map.Entry<Integer, TaskEntry>> tIter = TaskSet.iterator();
		// Go over all the Tasks in the TaskDB
		while(tIter.hasNext()){
			// retrieve the set element
			Map.Entry<Integer, TaskEntry> tMapEntry = tIter.next();
			// If this task is not an APPLICATION type
			if(tMapEntry.getValue().getContext().getType() != TaskType.APPLICATION)
				// skip it
				continue;
			
			// skip tasks from loacl machine
			if(tMapEntry.getValue().getAssignedServerID().equals(MachineDBLocalID))
				continue;
			
			// If tasks from that Machine shouldn't be taken
			if(TaskCountToBeTaken.get(tMapEntry.getValue().getAssignedServerID()).equals(0))
				// skip it
				continue;
			
			// put the task in the temp TaskDB list to be taken
			TasksToBeTaken.put(tMapEntry.getKey(), tMapEntry.getValue());
			// replace "Tasks to be taken" count of the machine with a decremented one.
			Integer TasksToBoTakenFromMachine = TaskCountToBeTaken.remove(tMapEntry.getValue().getAssignedServerID());
			TaskCountToBeTaken.put(tMapEntry.getValue().getAssignedServerID(), TasksToBoTakenFromMachine - 1);
			
			SA.AddLogEntry("CMM", "Info - Will Take over Application Task " + tMapEntry.getKey() +
					" from Machine " + tMapEntry.getValue().getAssignedServerID());
		}
		
		// Take ownership of the found Tasks
		// Convert TasksToBeTaken to a Set
		Set<Map.Entry<Integer, TaskEntry>> ToBeTakenSet = TasksToBeTaken.entrySet();
		tIter = ToBeTakenSet.iterator();
		// Go over all the tasks in the TasksToBeTaken database
		while(tIter.hasNext()){
			// retrieve the set element
			Map.Entry<Integer, TaskEntry> TaskMapEntry = tIter.next();
			
			// Create an "Application Migration" message
			ClusterMessage_AppMigrate msg = new ClusterMessage_AppMigrate();
			// set the TaskID entry of the message to the current Task
			msg.setTaskID(TaskMapEntry.getKey());
			try {
				// Send the message to the Cluster
				SendClusterMessage(msg);
			} catch (CMMExn_ClusterErr e) {
				// TODO remove StackPrint 
				e.printStackTrace();
				SA.AddLogEntry("CMM", "Error - Failed to send \"Application Migration Task\" message to the Cluster.");
				SA.UnrecoverableClusterError();
			}
		}
		
		
	}
*/
	
//	------------------------------------------------------------------------	
	
	private synchronized Integer FindLeastBusyServer() {
		
		// Transform the machine database from a TraaMap into a Set
		Set<Map.Entry<Integer, MachineEntry>> MachineSet = MachineDB.entrySet();
		// Create an Iterator to run through the Set
		Iterator<Map.Entry<Integer, MachineEntry>> it = MachineSet.iterator();
		int MinTaskAmount = Integer.MAX_VALUE;
		Integer SelectedMachineID = null;
		// Run througn the Set
		while(it.hasNext()){
			// Retrieve an entry from the Set
			Map.Entry<Integer, MachineEntry> MapEntry = it.next();
			// Retrieve the machine entry from the Set entry
			MachineEntry MEntry = MapEntry.getValue();
			// if the machine's task amount is the current minimum
			if(MEntry.getTaskCount() <= MinTaskAmount){
				// Set task count minimum to this machine's task count 
				MinTaskAmount = MEntry.getTaskCount();
				// Set MachineID 
				SelectedMachineID = MapEntry.getKey();
			}
		}
		
		return SelectedMachineID;
	}	
	
	/**
	 * This method creates a snapshot Task Database out of the CMM's regular
	 * Task Database, starting from (not including) StartingTaskID.
	 * 
	 * @param StartingTaskID - The minimum TaskID from which to start creating the 
	 * snapshot Task Database. If valuse is null, then the whole Task Database will
	 * be cloned.  
	 * @return The Snapshot Task Database
	 * @throws CloneNotSupportedException - Error while cloning TaskDB
	 */
	private synchronized TreeMap<Integer, TaskEntry> CreateSnapshotTaskDB(Integer StartingTaskID) throws CloneNotSupportedException {
		
		// Create a snapshot Task database
		TreeMap<Integer, TaskEntry> SnapTaskDB = new TreeMap<Integer, TaskEntry>();
		// Convert TaskDB from TreeMap into a Set
		Set<Map.Entry<Integer, TaskEntry>> TaskSet = TaskDB.entrySet();
		Iterator<Map.Entry<Integer, TaskEntry>> Tit = TaskSet.iterator();
		while(Tit.hasNext()){
			Map.Entry<Integer, TaskEntry> TMapEntry = Tit.next();
			Integer SnapKey = new Integer(TMapEntry.getKey());
			TaskEntry SnapEntry = null;
			try {
				SnapEntry = TMapEntry.getValue().clone();
			} catch (CloneNotSupportedException e) {
				// TODO remove stack print
				e.printStackTrace();
				throw e;
			}
			// If there's no TaskID bottom limit, or current TaskID is above the limit
			if((StartingTaskID == null) || (SnapKey.intValue() > StartingTaskID.intValue())){
				// put the new copy into the snapshot task database
				SnapTaskDB.put(SnapKey, SnapEntry);	
			}
		}
		return SnapTaskDB;
	}
	
//------------------------------------------------------------------------
//  Public Methods	
//------------------------------------------------------------------------
		
	
	/**
	 * The constructor of the Cluster Management Module. It initializes all
	 * the communication with the cluster and returns once the machine is
	 * synchronized with all the active machines in the cluster - 
	 * hence after training.
	 * 
	 * @param SA - A reference to the Server Application which activates the CMM 
	 * @param mySAInfo - The connection information of the SA for it's clients 
	 * @param ProxyConnInfo - The connection information for the Proxy Server
	 * @param ClusterGroupName - The Cluster group name for the CMM to connect to
	 * @param DefaultGatewayIP - The IP address of the default gateway.
	 * @param ProxyMask - the ip mask of the network (for the Proxy Server) 
	 * @param NetworkConnName - The name of the network connection of the Proxy Server.
	 * @param DNSIP - The IP address of the DNS server of the network
	 * @throws CMMExn_ClusterErr - An error occured while connecting to the Cluster
	 * @throws IOException - Failed to convert the dynamic IP into a static IP 
	 */
	public ClusterManagementModule(ServerApplication SA, ConnectionInformation mySAInfo, ConnectionInformation ProxyConnInfo, String ClusterGroupName, String DefaultGatewayIP, String ProxyMask, String NetworkConnName, String DNSIP, boolean Interactive) throws CMMExn_ClusterErr, IOException {
		
		this.SA = SA;
		GroupName = ClusterGroupName;
		
		LocalMachineEntry = new MachineEntry();
		LocalMachineEntry.putConnInfo(mySAInfo);
		LocalMachineEntry.putMachineState(MachineState.PRE_TRAINING);
		LocalMachineEntry.putTaskCount(0);
		try {
			LocalMachineEntry.putClusterMachineName(InetAddress.getLocalHost().getHostName());
		} catch (UnknownHostException e2) {
			// TODO Auto-generated catch block
			e2.printStackTrace();
			throw e2;
		}
		InteractiveTrainingMode = Interactive; 
		this.ProxyMask = ProxyMask; 
		this.DefaultGatewayIP = DefaultGatewayIP;
		NetworkConnectionName = NetworkConnName;
		
		// Initialize the task database
		TaskDB = new TreeMap<Integer, TaskEntry>();
		
		// Initialize the machine database
		MachineDB = new TreeMap<Integer, MachineEntry>();
		
		// Initialize the Snapshot TaskDB List
		ToBeMentoredList = new LinkedList<ToBeMentoredEntry>();
		
		MentorList = new LinkedList<Mentor>();
		
		SA.AddLogEntry("CMM", "Info - CMM Started.");

		// Wait for 2 seconds for the local IP address to change to static
		try {
			Thread.sleep(3*1000);
		} catch (InterruptedException e3) {
			// TODO Auto-generated catch block
			e3.printStackTrace();
		}
		
		
		SA.AddLogEntry("CMM", "Info - Trying to connect to the cluster via Spread");
		
		// Start Spread connection
		boolean priority = false;
		boolean groupMembership = true;
		
		ClusterConnection = new SpreadConnection();
		
		// private connection name is set to the name of the local machine
		String privateName = LocalMachineEntry.getClusterMachineName();
		
		synchronized(this){
		
			try {
				SA.AddLogEntry("CMM", "Info - Attempting to connect local Spread Deamon.");
				ClusterConnection.connect(null, 0, privateName, priority, groupMembership);
			} catch (SpreadException e) {
				// TODO Remove stack print
				e.printStackTrace();
				SA.AddLogEntry("CMM", "Error - Failed to establish Spread connection with deamon on local machine.");
				throw new CMMExn_ClusterErr();
			}
			
			try {	
				// Create and add a new Spread listener for the spread
				SpreadListener = new AdvancedSpreadListener(this, LocalMachineEntry.getClusterMachineName(), LocalMachineEntry.getMachineState());
				ClusterConnection.add(SpreadListener);
			
				// Join the server group in the cluster
				ClusterGroup = new SpreadGroup();
				ClusterGroup.join(ClusterConnection, GroupName);
			
			
			} catch (SpreadException e) {
				// TODO Remove StackPrint
				e.printStackTrace();
				SA.AddLogEntry("CMM", "Error - Failed to connect to the cluster. Not running CMM thread.");
				CMMExn_ClusterErr exn = new CMMExn_ClusterErr();
				throw exn;
			}
		
			SA.AddLogEntry("CMM", "Info - Spread connection to the cluster established.");
		
			try {
				// wait for the training to end (or to be skipped)
				wait();
			} catch (InterruptedException e2) {
			}
			SA.AddLogEntry("CMM", "Info - CMM now has the full Cluster state. Sending \"New Server Ready\" message to the Cluster.");
		}
		
		// The CMM is now ready for steady state activity

		if(InteractiveTrainingMode)
			WaitForEnter("Press Enter to send \"New Server Ready\" message to the Cluster.");
		
		synchronized(this){
			// Send message to the cluster - "New Server Ready"
			ClusterMessage_NewServerReady msg = new ClusterMessage_NewServerReady();
			// Put local machine details into the message
			LocalMachineEntry.putMachineState(MachineState.ACTIVE);
			SpreadListener.setLocalMachineState(LocalMachineEntry.getMachineState());
			msg.setNewMachineEntry(LocalMachineEntry);
			try {
				SendClusterMessage(msg);
			} catch (CMMExn_ClusterErr e) {
				SA.AddLogEntry("CMM", "Error - Failed to send \"New Server Ready\" message to the Cluster.");
				throw e;
			}
	
			try {
				wait();
			} catch (InterruptedException e2) {
			}
			SA.AddLogEntry("CMM", "Info - The local Machine has succesfully joined the Cluster, has been Trained, and became Active");
		}
				
		// -- Proxy Initialization --
		// Proxy TaskID is always 0 as this is the first Task 
		if(MachineDBLocalID.equals(new Integer(0))){
			try {
				Proxy = new ProxyServer(this, ProxyConnInfo, ProxyMask, DefaultGatewayIP, NetworkConnectionName);
				SA.AddLogEntry("CMM", "Info - Proxy Server initialized.");
			} catch (IOException e1) {
				// TODO remove StackPrint
				e1.printStackTrace();
				SA.AddLogEntry("CMM", "Error - Failed to initialize the Proxy Server.");
				Close();
				CMMExn_ClusterErr exn = new CMMExn_ClusterErr();
				throw exn;
			}
		
			try{
				// Send "Proxy New Task" message to the Cluster
				ProxyNewTask(ProxyConnInfo);
			} catch (CMMExn_ClusterErr e){
				SA.AddLogEntry("CMM", "Error - Failed to send \"Proxy New Task\" message to the Cluster.");
				try {
					Proxy.Close();
					SA.AddLogEntry("CMM", "Info - Terminated Proxy Server.");
				} catch (IOException e1) {
					// TODO remove StackTrace
					e1.printStackTrace();
				}
				throw e;
			}
		}
	}

//	------------------------------------------------------------------------
//  ProxyServer Interface	
//	------------------------------------------------------------------------	
	
	public synchronized ConnectionInformation FindLeastBusyServerInfo() {

		SA.AddLogEntry("CMM", "Info - Received a \"Find Least Busy Server\" request from the Proxy server.");
		
		Integer SelectedMachineID = FindLeastBusyServer();
		
		SA.AddLogEntry("CMM", "Info - Chose least busy server - MachineID " + SelectedMachineID);
		
		return MachineDB.get(SelectedMachineID).getConnInfo();
	}

//	------------------------------------------------------------------------	
	
	public void AddLogEntry(String ModuleName, String Entry) {

		SA.AddLogEntry(ModuleName, Entry);
	}

//	------------------------------------------------------------------------
//  Mentor Interface	
//	------------------------------------------------------------------------	
	
	public void SendTrainingMsg(ClusterMessage_TrainingUpdateTask Message) throws CMMExn_ClusterErr {
		
		try {
			SendClusterMessage(Message);
		} catch (CMMExn_ClusterErr e) {
			// TODO remove stack print
			e.printStackTrace();
			SA.AddLogEntry("CMM", "Error - Failed to send a training message to the cluster.");
			SA.UnrecoverableClusterError();
			throw e;
		}
	}
	
//	------------------------------------------------------------------------	
	
	public void SendTrainingMsg(ClusterMessage_TrainingTerminateTask Message) throws CMMExn_ClusterErr {
		
		try {
			SendClusterMessage(Message);
		} catch (CMMExn_ClusterErr e) {
			// TODO remove stack print
			e.printStackTrace();
			SA.AddLogEntry("CMM", "Error - Failed to send a training termination message to the cluster.");
			SA.UnrecoverableClusterError();
			throw e;
		}
	}
	
//	------------------------------------------------------------------------
//  ServerApplication Interface	
//	------------------------------------------------------------------------	

	public void AppNewTask(SAGenTaskContext Context) throws CMMExn_ClusterErr {

		SA.AddLogEntry("CMM", "Info - Recieved a \"New Task\" request from the SA.");
		
		// Create a "New Task" message 
		ClusterMessage_AppNewTask msg = new ClusterMessage_AppNewTask();
		msg.setTaskContext(Context);
		
		// Send the message through the cluster
		try {
			SendClusterMessage(msg);
		} catch (CMMExn_ClusterErr e) {
			SA.AddLogEntry("CMM", "Error - Failed to send \"New Task\" message to the cluster.");
			throw e;
		}
	}

//------------------------------------------------------------------------	
	
	public synchronized void AppTaskContextUpdate(Integer TaskID,
			SAGenTaskContext ContextChange) throws CMMExn_TaskErr, CMMExn_ClusterErr {

		SA.AddLogEntry("CMM", "Info - Recieved a \"Task Context Update\" request from the SA for TaskID " + TaskID);
		
		// This past is synchronized because 
		// access to CMM databases is restricted 

		// Check if task exists in task database
		if(!TaskDB.containsKey(TaskID)){
			SA.AddLogEntry("CMM", "Error - TaskID " + TaskID + " does not exist in the Task Database.");
			CMMExn_TaskErr TaskErr = new CMMExn_TaskErr();
			throw TaskErr;
		}
		
		// Check if task belongs to local machine
		TaskEntry Task = TaskDB.get(TaskID);
		if(!Task.getAssignedServerID().equals(MachineDBLocalID)){
			SA.AddLogEntry("CMM", "Error - Task's assigned Server (" + Task.getAssignedServerID() 
					+ ") does not match local MachineID (" + MachineDBLocalID +")");
			CMMExn_TaskErr TaskErr = new CMMExn_TaskErr();
			throw TaskErr;
		}
		
		// Check if task type is APPLICATION
		if(Task.getContext().getType()!=TaskType.APPLICATION){
			SA.AddLogEntry("CMM", "Error - Task " + TaskID + " is not of APPLICATION type.");
			// Application is not allowed to update non-application tasks.
			throw new CMMExn_TaskErr();
		}
				
		// Create an "Update Task" message
		ClusterMessage_AppUpdateTask msg = new ClusterMessage_AppUpdateTask();
		msg.setTaskID(TaskID);
		msg.setContextUpdate(ContextChange);
		
		// Send the message through the cluster
		try {
			SendClusterMessage(msg);
		} catch (CMMExn_ClusterErr e) {
			SA.AddLogEntry("CMM", "Error - Failed to send \"Update Task\" message to the cluster." +
					" TaskID : " + TaskID);
			throw e;
		}
	}

//------------------------------------------------------------------------	
	
	public synchronized void AppTaskFinish(Integer TaskID) throws CMMExn_TaskErr, CMMExn_ClusterErr {
		
		SA.AddLogEntry("CMM", "Info - Recieved a \"Task Finish\" request for TaskID " + TaskID);
		
		// This past is synchronized because 
		// access to CMM databases is restricted 
	
		// Check if task exists in task database
		if(!TaskDB.containsKey(TaskID)){
			SA.AddLogEntry("CMM", "Error - TaskID " + TaskID + " does not exist in the Task Database.");
			CMMExn_TaskErr TaskErr = new CMMExn_TaskErr();
			throw TaskErr;
		}

		// Check if task belongs to local machine
		TaskEntry Task = TaskDB.get(TaskID);
		if(!Task.getAssignedServerID().equals(MachineDBLocalID)){
			SA.AddLogEntry("CMM", "Error - Task's assigned Server (" + Task.getAssignedServerID() 
					+ ") does not match local MachineID (" + MachineDBLocalID +")");
			CMMExn_TaskErr TaskErr = new CMMExn_TaskErr();
			throw TaskErr;
		}

		// Check if task type is APPLICATION
		if(Task.getContext().getType()!=TaskType.APPLICATION){
			SA.AddLogEntry("CMM", "Error - Task " + TaskID + " is not of APPLICATION type.");
			// Application is not allowed to terminate non-application tasks.
			throw new CMMExn_TaskErr();
		}
		
		// Create a "Terminate Task" message
		ClusterMessage_AppTerminateTask msg = new ClusterMessage_AppTerminateTask();
		msg.setTaskID(TaskID);
		
		// Send the message through the cluster
		try {
			SendClusterMessage(msg);
		} catch (CMMExn_ClusterErr e) {
			SA.AddLogEntry("CMM", "Error - Failed to send \"Terminate Task\" message to the cluster." +
					" TaskID : " + TaskID);
			throw e;
		}
	}

//	------------------------------------------------------------------------	
	
	public void Close() {
	
		SA.AddLogEntry("CMM", "Info - Received a \"Close\" instruction.");
		
		// Close Proxy Server
		if(ProxyMachineID.equals(MachineDBLocalID)){
			try {
				Proxy.Close();
			} catch (IOException e1) {
				// TODO remove StackTrace
				e1.printStackTrace();
				SA.AddLogEntry("CMM", "Error - Failed to close Proxy Server.");
			}
		}
		
		// Close Mentor Threads
		// For each mentor in the mentors list
		while(!MentorList.isEmpty()){
			// Remove the first mentor in the list
			Mentor mentor = MentorList.removeFirst();
			// Close the mentor
			// note: The mentor may be finished, and the thread may be long dead
			mentor.Close();
		}
		
		// Close Spread Connection
		try {	
			// Remove listener from the connection
			ClusterConnection.remove(SpreadListener);
			// Leave the Cluster group
			ClusterGroup.leave();
			// Disconnect from the Spread connection
			ClusterConnection.disconnect();
			SA.AddLogEntry("CMM", "Info - Disconnected Spread connection to the Cluster.");
		} catch (SpreadException e) {
			e.printStackTrace();
			SA.AddLogEntry("CMM", "Error - Failed to close the Spread connection to the Cluster.");
		}
	}
	
//	------------------------------------------------------------------------
//  AdvancedSpreadListener Interface	
//	------------------------------------------------------------------------	

	public synchronized void HandleNetworkEvent(SpreadGroup[] StayedMachines){
		
		// Convert MachineDB from TreeMap into a Set
		Set<Map.Entry<Integer, MachineEntry>> MachineSet = MachineDB.entrySet();
		Iterator<Map.Entry<Integer, MachineEntry>> it = MachineSet.iterator();
		// for each element in the set
		while(it.hasNext()){
			// retrieve the set element
			Map.Entry<Integer, MachineEntry> MapEntry = it.next();
			boolean found = false;
			// search for the machine in the StayedMachines array
			for(SpreadGroup Machine:StayedMachines){
				String MachineName = AdvancedSpreadListener.ConvertFromSpreadName(Machine.toString());
				if(MachineName.equals(MapEntry.getValue().getClusterMachineName())){	
					found = true;
					break;
				}
			}
			// if machine not found
			if(!found){
				// this is the machine that crushed
				ReceiveClusterLeave(MapEntry.getValue().getClusterMachineName());
				break;
			}
		}
			
	}
	
//	------------------------------------------------------------------------	
	
	public void ReceiveLocalFirstJoin() {
		
		SA.AddLogEntry("CMM", "Info - Received self joining to cluster message.");
		SA.AddLogEntry("CMM", "Info - This machine is the first in the Cluster - skipping training.");
		
		synchronized (this) {
			// Wake the constructor thread
			notifyAll();
		}
	}

//	------------------------------------------------------------------------	
	
	public void ReceiveLocalClusterJoin() {
		
		SA.AddLogEntry("CMM", "Info - Received self joining to cluster message. Waiting for a training session.");
		
//		LocalMachineEntry.putMachineState(MachineState.TRAINING);
//		SpreadListener.setLocalMachineState(LocalMachineEntry.getMachineState());
	}
	
//	------------------------------------------------------------------------	
	
	public void ReceiveNewMachineJoin(String MachineName) {
		
		// Hote: this message is NOT pushed into the TrainingMessageQueue
		//       for training machines		
		if(LocalMachineEntry.getMachineState()==MachineState.ACTIVE){
			SA.AddLogEntry("CMM", "Info - Received a \"Membership Join\" message from the cluster. Machine " + MachineName + " has joined the cluster.");

			// Decide on who is going to mentor the joined machine.
			// If not going to be the Mentor - nothing left to do.
			if(!MachineDBLocalID.equals(FindLeastBusyServer()))
				return;
		
			// Create a snapshot Machine database
			TreeMap<Integer, MachineEntry> SnapMachineDB = new TreeMap<Integer, MachineEntry>();
			TreeMap<Integer, TaskEntry> SnapTaskDB = null;
			synchronized(this){
				// Convert MachineDB from TreeMap into a Set
				Set<Map.Entry<Integer, MachineEntry>> MachineSet = MachineDB.entrySet(); 
				Iterator<Map.Entry<Integer, MachineEntry>> it = MachineSet.iterator();
				// for each element in the set
				while(it.hasNext()){
					// retrieve the set element
					Map.Entry<Integer, MachineEntry> MapEntry = it.next();
					// Create a copy of the Machine ID
					Integer SnapKey = new Integer(MapEntry.getKey());
					// Create a copy of the MachineEntry
					MachineEntry SnapEntry = null;
					try {
						SnapEntry = MapEntry.getValue().clone();
					} catch (CloneNotSupportedException e) {
						// TODO remove stack print
						e.printStackTrace();
						SA.AddLogEntry("CMM", "Error - Failed to create a Snapshot Machine database.");
						SA.UnrecoverableClusterError();
						return;
					}
					
					// Reset the Task Count of the Machine.
					// The Trainee will learn the correct Task Count via "Training Update" messages.
					// note: this is a bug fix. Otherwise, Trainee gets TaskCount mixed up after
					// 		 Training Migration.
					SnapEntry.putTaskCount(0);
				
					// put the new copy into the snapshot machine database
					SnapMachineDB.put(SnapKey, SnapEntry);
				}
				

				try {
					// Create a snapshot Task database
					SnapTaskDB = CreateSnapshotTaskDB(null);
				} catch (CloneNotSupportedException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
					SA.AddLogEntry("CMM", "Error - Failed to create a Snapshot Task database.");
					SA.UnrecoverableClusterError();
					return;
				}
			}
		
			if(InteractiveTrainingMode)
				WaitForEnter("Press Enter to send \"Training New Task\" message to " + MachineName);
			
			// Start mentoring the new machine
			SA.AddLogEntry("CMM", "Info - Started mentoring new machine " + MachineName);
			// Create a "Training New Task" message
			MachineTrainingData data = new MachineTrainingData();
			data.setMachineName(MachineName);
			data.setMachineDB(SnapMachineDB);
			ClusterMessage_TrainingNewTask msg = new ClusterMessage_TrainingNewTask();
			msg.setContext(data);
			try {
				// Send the message to the Cluster
				SendClusterMessage(msg);
			} catch (CMMExn_ClusterErr e) {
				// TODO remove stack trace
				e.printStackTrace();
				SA.AddLogEntry("CMM", "Error - Failed to send a \"Training New Task\" message.");
				SA.UnrecoverableClusterError();
				return;
			}
		
			// Put Snapshot TaskDatabase into the Snapshot TaskDB List
			ToBeMentoredList.addLast(new ToBeMentoredEntry(MachineName, SnapTaskDB));
		}
	}
	
//	------------------------------------------------------------------------
	
	public synchronized void ReceiveClusterLeave(String LeavingMachineName) {
		
		SA.AddLogEntry("CMM", "Info - Received a \"Membership Leave\" message from the Cluster. Machine " + LeavingMachineName + " has left the Cluster.");
		
		if(LocalMachineEntry.getMachineState()==MachineState.PRE_TRAINING){
			return;
		}
	
		Integer LeaverMachineID = null;
		// Convert MachineDB from TreeMap into a Set
		Set<Map.Entry<Integer, MachineEntry>> MachineSet = MachineDB.entrySet();
		Iterator<Map.Entry<Integer, MachineEntry>> it = MachineSet.iterator();
		// for each element in the set
		while(it.hasNext()){
			// retrieve the set element
			Map.Entry<Integer, MachineEntry> MapEntry = it.next();
			// If machine name matches the leaving machine
			if(MapEntry.getValue().getClusterMachineName().equals(LeavingMachineName)) {
				LeaverMachineID = MapEntry.getKey();
				break;
			}
		}
		if(LeaverMachineID != null){
			// If in TRAINING state
			if(LocalMachineEntry.getMachineState()==MachineState.TRAINING){
				// Create a dummy "Remove Machine" message
				ClusterMessage_RemoveMachine Rm_Msg = new ClusterMessage_RemoveMachine();
				Rm_Msg.setSenderID(LeaverMachineID);
				// Put the message in the Training Message Queue
				StoreMessageWhileTraining(Rm_Msg);
				return;
			}
			
			// remove the machine from the Machine Database
			RemoveMachineFromCluster(LeaverMachineID);
		}
		else {
			// This is a viable situation. This can happen if a machine
			// leaves the cluster before it completes it's training and 
			// becomes ACTIVE 
			SA.AddLogEntry("CMM", "Warning - No Machine was found with the machine name - " + LeavingMachineName + " in the Machine Database.");
			return;
		}
		
		// Execute Fault Tolerance
		ExecuteFaultTolerance(LeaverMachineID);
		
		return;
	}
	
//	------------------------------------------------------------------------	
//	------------------------------------------------------------------------
	
	public synchronized void ReceiveAppNewTask(Integer MachineID, SAGenTaskContext Context) {
	
		SA.AddLogEntry("CMM", "Info - Recieved an \"Application New Task\" message from the cluster." +
				" Sender ID - " + MachineID);
		
		// If local machine is PRE_TRAINING - ignore the message.
		if(LocalMachineEntry.getMachineState()==MachineState.PRE_TRAINING)
			return;
		
		Integer NewTaskID = null;
		try {
			NewTaskID = InsertNewTaskIntoDB(MachineID, Context);
		} catch (CMMExn_TaskErr e1) {
			SA.AddLogEntry("CMM", "Error - Cluster message contains an invalid MachineID.");
			SA.UnrecoverableClusterError();
			return;
		}
		
		SA.AddLogEntry("CMM", "Info - Added the new application task to the Task Database with TaskID=" + NewTaskID);
		
		// If message originated on this machine
		if(MachineID.equals(MachineDBLocalID)){
			try {
				// Report the New Task ID to the SA
				SA.ReceiveNewTaskId(NewTaskID);
			} catch (Exception e) {
				SA.AddLogEntry("CMM", "Error - SA did not expect to recieve New Task ID. Removing task from Database");
				e.printStackTrace();
				try {
					// Send "Application Task Finish" message to the Cluster
					AppTaskFinish(NewTaskID);
				} catch (Exception e1) {
					// TODO remove stack print
					e1.printStackTrace();
					SA.AddLogEntry("CMM", "Error - Could not send \"Application Task Finish\" message to the Cluster.");
				}
				SA.UnrecoverableClusterError();
			}
			LocalMachineEntry.incTaskCount();
		}
	}

//	------------------------------------------------------------------------	
	
	public synchronized void ReceiveAppUpdateTask(Integer MachineID, Integer TaskID, SAGenTaskContext ContextUpdate) {

		SA.AddLogEntry("CMM", "Info - Recieved an \"Application Update Task\" message from the cluster." +
				" Sender ID - " + MachineID + " , TaskID - " + TaskID);
		
		if(!MachineDB.containsKey(MachineID)){
			SA.AddLogEntry("CMM", "Error - Sender MachineID " + MachineID + " does not exist in the Machine Database.");
			SA.UnrecoverableClusterError();
			return;
		}
		if(!TaskDB.containsKey(TaskID)){
			SA.AddLogEntry("CMM", "Error - TaskID " + TaskID + " does not exist in the Task Database.");
			SA.UnrecoverableClusterError();
			return;
		}
			
		// Retrieve task entry from task database
		TaskEntry TEntry = TaskDB.get(TaskID);
		// Verify that task belongs to the correct machine
		if(!TEntry.getAssignedServerID().equals(MachineID)){
			SA.AddLogEntry("CMM", "Error - Task's assigned Server (" + TEntry.getAssignedServerID() 
					+ ") does not match sender MachineID (" + MachineID +")");
			SA.UnrecoverableClusterError();
			return;
		}
		// Retrieve task context from task entry
		TaskContext Context = TEntry.getContext();
		// Verify that task context type is APPLICATION
		if(Context.getType()!=TaskType.APPLICATION){
			SA.AddLogEntry("CMM", "Error - Task type is " + 
					Context.getType() + " instead of APPLICATION.");
			SA.UnrecoverableClusterError();
			return;
		}
		// Convert to SA task context
		SAGenTaskContext OldSAContext = (SAGenTaskContext)Context;
		// Reconstruct task's context from old context and incramental change
		SAGenTaskContext NewSAContext = SA.ReconstructTaskContext(OldSAContext, ContextUpdate);
		// Put reconstructed context into the task's entry
		TEntry.putContext(NewSAContext);
	}
		
//	------------------------------------------------------------------------	
	
	public synchronized void ReceiveAppTerminateTask(Integer MachineID, Integer TaskID) {

		SA.AddLogEntry("CMM", "Info - Recieved a \"Application Terminate Task\" message from the cluster." +
				" Sender ID - " + MachineID + " , TaskID - " + TaskID);

		if(!MachineDB.containsKey(MachineID)){
			SA.AddLogEntry("CMM", "Error - Sender MachineID " + MachineID + " does not exist in the Machine Database.");
			SA.UnrecoverableClusterError();
			return;
		}
		if(!TaskDB.containsKey(TaskID)){
			SA.AddLogEntry("CMM", "Error - TaskID " + TaskID + " does not exist in the Task Database.");
			SA.UnrecoverableClusterError();
			return;
		}
			
		// Retrieve task's entry from the task database
		TaskEntry TEntry = TaskDB.get(TaskID);
		// Verify that the task belongs to the correct machine
		if(!TEntry.getAssignedServerID().equals(MachineID)){
			SA.AddLogEntry("CMM", "Error - Task's assigned Server (" + TEntry.getAssignedServerID() 
					+ ") does not match sender MachineID (" + MachineID +")");
			SA.UnrecoverableClusterError();
			return;
		}
		// Remove task's entry from the task database
		TaskDB.remove(TaskID);
		// Retrieve machine's entry from the machine database
		MachineEntry MEntry = MachineDB.get(MachineID);
		// Decrament the task count of the machine
		MEntry.decTaskCount();
		
		if(LocalMachineEntry.getMachineState()==MachineState.ACTIVE){
			// If the task was on this machine
			if(MachineDBLocalID.equals(MachineID)) {
				LocalMachineEntry.decTaskCount();
			}
		}
		
		ClusterTaskCount--;
	}
	
//	------------------------------------------------------------------------	
	
	public synchronized void ReceiveAppMigrate(Integer MachineID, Integer TaskID) {
		
		SA.AddLogEntry("CMM", "Info - Received an \"Application Migrate Task\" message from MachineID="
				+ MachineID + " for TaskID=" + TaskID);
		
		// check that the MachineID is valid
		if(!MachineDB.containsKey(MachineID)){
			SA.AddLogEntry("CMM", "Error - MachineID=" + MachineID + " does not exist in the Machine Database.");
			SA.UnrecoverableClusterError();
			return;
		}
		// check that the TaskID is valid
		if(!TaskDB.containsKey(TaskID)){
			SA.AddLogEntry("CMM", "Error - TaskID=" + TaskID + "does not exist in the TaskDatabase.");
			SA.UnrecoverableClusterError();
			return;
		}
		TaskEntry MigratingTask = TaskDB.get(TaskID);
		// check that the Task is of APPLICATION type
		if(MigratingTask.getContext().getType() != TaskType.APPLICATION){
			SA.AddLogEntry("CMM", "Error - Task " + TaskID + " is not of APPLICATION type.");
			SA.UnrecoverableClusterError();
			return;
		}
		
		
		// Find the previous owner of the task
		Integer PrevOwner = MigratingTask.getAssignedServerID();
		// if the previous owner of the task is still in the Machine Database
		if((PrevOwner != null) && (MachineDB.containsKey(PrevOwner))){
			// decrese it's task count 
			MachineDB.get(PrevOwner).decTaskCount();
		}
		// Assign the Task the new Machine
		MigratingTask.putAssignedServerID(MachineID);
		// Increase the task count of the receiving Machine
		MachineDB.get(MachineID).incTaskCount();
		
		SA.AddLogEntry("CMM", "Info - Moved Application Task " + TaskID + " to the ownership of Machine " + MachineID);
		
		// The following are checks of ownership of the tasks.
		// they should be executed only if the Machine is in Active State
		if(LocalMachineEntry.getMachineState() != MachineState.ACTIVE)
			return;
		
		// If the message originated from this machine.
		// This machine should take the ownership of the Task.
		if(MachineID.equals(MachineDBLocalID)){
			SA.AddLogEntry("CMM", "Info - Task " + TaskID + " is being transfered to this Machine. Notifying the Server Application.");
			// Instruct the SA to start servicing this Task
			SA.StartServicingTask(TaskID, (SAGenTaskContext)MigratingTask.getContext());
			LocalMachineEntry.incTaskCount();
			
			SA.AddLogEntry("CMM", "Info - Task " + TaskID + " has successfully migrated to this Machine.");
			return;
		}
		// If the Task was previously serviced by this Machine
		// This Machine should stop servicing it.
		// note: PrevOwner may be null in case of Machine leave
		if((PrevOwner != null) && (MachineDBLocalID.equals(PrevOwner))){
			SA.AddLogEntry("CMM", "Info - Task " + TaskID + " is being transfered from this Machine. Notifying the Server Application.");
			// Instruct the SA to stop servicing this Task
			SA.StopServicingTask(TaskID);
			LocalMachineEntry.decTaskCount();
			
			SA.AddLogEntry("CMM", "Info - Task " + TaskID + " has successfully migrated from this Machine.");
			return;
		}
		
	}

//	------------------------------------------------------------------------
	
	public synchronized void ReceiveAppTransferTask(Integer SenderID, Integer TaskID, Integer NewOwnerID) {

		SA.AddLogEntry("CMM", "Info - Recieved a \"Application Transfer Task\" message from MachineID " + SenderID +
				" for TaskID " + TaskID + " and new owner MachineID " + NewOwnerID);
		
		if(!MachineDB.containsKey(SenderID)){
			SA.AddLogEntry("CMM", "Error - Sender MachineID=" + SenderID + " does not exist in the Machine Database.");
			SA.UnrecoverableClusterError();
			return;
		}
		
		if(!MachineDB.containsKey(NewOwnerID)){
			SA.AddLogEntry("CMM", "Error - New owner MachineID=" + NewOwnerID + " does not exist in the Machine Database.");
			SA.UnrecoverableClusterError();
			return;
		}
		
		if(!TaskDB.containsKey(TaskID)){
			SA.AddLogEntry("CMM", "Error - TaskID=" + TaskID + " does not exist in the TaskDatabase.");
			SA.UnrecoverableClusterError();
			return;
		}
		
		if(!TaskDB.get(TaskID).getContext().getType().equals(TaskType.APPLICATION)){
			SA.AddLogEntry("CMM", "Error - TaskID=" + TaskID + " is not of type APPLICATION.");
			SA.UnrecoverableClusterError();
			return;
		}
		
		if(!TaskDB.get(TaskID).getAssignedServerID().equals(SenderID)){
			SA.AddLogEntry("CMM", "Error - Sender MachineID=" + SenderID + " is not consistent with Task's current assigned server " + 
					TaskDB.get(TaskID).getAssignedServerID());
			SA.UnrecoverableClusterError();
			return;
		}
		
		// Assign new server to the Task
		TaskDB.get(TaskID).putAssignedServerID(NewOwnerID);
		// decrease task count of the old task owner
		MachineDB.get(SenderID).decTaskCount();
		// increase task count of the new task owner
		MachineDB.get(NewOwnerID).incTaskCount();
		
		if(NewOwnerID.equals(MachineDBLocalID)){
			LocalMachineEntry.incTaskCount();
			SA.AddLogEntry("CMM", "Info - Task " + TaskID + " is being transfered to this Machine. Notifying the Server Application.");
			SA.StartServicingTask(TaskID, (SAGenTaskContext)TaskDB.get(TaskID).getContext());
		}
		if(SenderID.equals(MachineDBLocalID)){
			LocalMachineEntry.decTaskCount();
		}
		// note: no need to notify SA if local machine is the sender, 
		// as this is done by the Load Balancing algorythm.
	}
	
//	------------------------------------------------------------------------	

	public void ReceiveNewServerReady(MachineEntry newMachineEntry) {

		Integer NewMachineID = null;
		synchronized(this){
			SA.AddLogEntry("CMM", "Info - Received a \"New Server Ready\" message from the cluster." +
					" Sender Name - " + newMachineEntry.getClusterMachineName());

			// If the local machine is in PRE_TRAINING state - do not insert the 
			// new MachineEntry to the Machine database
			if(LocalMachineEntry.getMachineState() == MachineState.PRE_TRAINING)
				return;

			// Get max MachineID from the machine database
			Integer MaxMachineID;
			if(MachineDB.isEmpty())
				MaxMachineID = new Integer(-1);
			else
				MaxMachineID = MachineDB.lastKey();

			// New machine ID is max+1 
			NewMachineID = new Integer(MaxMachineID.intValue() + 1);

			SA.AddLogEntry("CMM", "Info - The new MachineID allocated for the new machine " + newMachineEntry.getClusterMachineName() + " is " + NewMachineID);

			// Put the new machine entry into the machine database
			MachineDB.put(NewMachineID, newMachineEntry);

			// A new machine has become ACTIVE, so inc the count.
			ClusterActiveMachineCount++;

			// If this message originated on this machine
			if(newMachineEntry.getClusterMachineName().equals(LocalMachineEntry.getClusterMachineName())) {
				// Update the local MachineID
				MachineDBLocalID = NewMachineID;

				notifyAll();
			}
		}
		
		// As a new Machine joins the Cluster execute Load Balancing,
		// transfering some of own APPLICATION tasks to it. 
		ExecuteLoadBalancing(NewMachineID);
	}

//	------------------------------------------------------------------------	
	
	public void ReceiveUnsupportedMessage() {
		SA.AddLogEntry("CMM", "Error - Received an unsupported message from the cluster.");
		SA.UnrecoverableClusterError();
	}

//	------------------------------------------------------------------------	
	
	public synchronized void ReceiveProxyNewTask(Integer MachineID, ProxyContext Context) {
		
		SA.AddLogEntry("CMM", "Info - Recieved a \"Proxy New Task\" message from the cluster." +
				" Sender ID - " + MachineID);
		
		Integer NewTaskID = null;
		try {
			NewTaskID = InsertNewTaskIntoDB(MachineID, Context);
		} catch (CMMExn_TaskErr e1) {
			SA.AddLogEntry("CMM", "Error - Error in Cluster message - dropping the message.");
			return;
		}
		
		SA.AddLogEntry("CMM", "Info - Added a new Proxy Server task with TaskID=" + NewTaskID);
		
		if(ProxyMachineID != null){
			SA.AddLogEntry("CMM", "Error - Recieved a \"Proxy New Task\" message while Proxy Server is already active.");
			SA.UnrecoverableClusterError();
		}
		else
			ProxyMachineID = MachineID;
		
		LocalMachineEntry.incTaskCount();
	}
	
//	------------------------------------------------------------------------	
	
	public synchronized void ReceiveProxyMigrate(Integer MachineID, Integer TaskID) {
		
		SA.AddLogEntry("CMM", "Info - Received an \"Proxy Migrate Task\" message from MachineID="
				+ MachineID + " for TaskID=" + TaskID);
		
		// check that the MachineID is valid
		if(!MachineDB.containsKey(MachineID)){
			SA.AddLogEntry("CMM", "Error - MachineID=" + MachineID + " does not exist in the Machine Database.");
			SA.UnrecoverableClusterError();
			return;
		}
		// check that the TaskID is valid
		if(!TaskDB.containsKey(TaskID)){
			SA.AddLogEntry("CMM", "Error - TaskID=" + TaskID + "does not exist in the TaskDatabase.");
			SA.UnrecoverableClusterError();
			return;
		}
		TaskEntry MigratingTask = TaskDB.get(TaskID);
		// check that the Task is of PROXY type
		if(MigratingTask.getContext().getType() != TaskType.PROXY){
			SA.AddLogEntry("CMM", "Error - Task " + TaskID + " is not of PROXY type.");
			SA.UnrecoverableClusterError();
			return;
		}
		
		// Assign the Task the new Machine
		MigratingTask.putAssignedServerID(MachineID);
		// Increase the task count of the receiving Machine
		MachineDB.get(MachineID).incTaskCount();
		// Update the ProxyMachineID entry
		ProxyMachineID = MachineID;
		
		SA.AddLogEntry("CMM", "Info - Moved Proxy Task " + TaskID + " to the ownership of Machine " + MachineID);
	
		// Task ownership tests should be performed only if machine is active
		if(LocalMachineEntry.getMachineState() != MachineState.ACTIVE)
			return;
		
		// If the message originated from this machine.
		// This machine should take the ownership of the Task.		
		if(MachineDBLocalID.equals(MachineID)){
			SA.AddLogEntry("CMM", "Info - Proxy Task " + TaskID + " is being transfered to this Machine.");

			try {
				Proxy = new ProxyServer(this, ((ProxyContext)TaskDB.get(TaskID).getContext()).getConnInfo(), ProxyMask, DefaultGatewayIP, NetworkConnectionName);
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
				SA.AddLogEntry("CMM", "Error - Failed to initialize Proxy Server on local machine.");
				SA.UnrecoverableClusterError();
			}
			LocalMachineEntry.incTaskCount();
			
			SA.AddLogEntry("CMM", "Info - Proxy Task " + TaskID + " has successfully migrated to this Machine.");
			return;
		}
		
	}
	
//	------------------------------------------------------------------------
	
	public synchronized void ReceiveTrainingNewTask(Integer MachineID, MachineTrainingData Data){

		SA.AddLogEntry("CMM", "Info - Received a \"Training New Task\" message from the cluster." +
				" SenderID=" + MachineID + " for Trainee machine name - " + Data.getMachineName());
		
		// If local machine is in PRE_TRAINING state
		if(LocalMachineEntry.getMachineState() == MachineState.PRE_TRAINING) {
			// Check if the message is for this local machine
			if(LocalMachineEntry.getClusterMachineName().equals(Data.getMachineName())){
				SA.AddLogEntry("CMM", "Info - Beginning the Training session.");
				
				MachineDB = Data.getMachineDB();
				
				// Interpret the number of ACTIVE machines in the Cluster from the MachineDB
				// Convert MachineDB from TreeMap into a Set
				Set<Map.Entry<Integer, MachineEntry>> MachineSet = MachineDB.entrySet(); 
				Iterator<Map.Entry<Integer, MachineEntry>> it = MachineSet.iterator();
				// for each element in the set
				while(it.hasNext()){
					// retrieve the set element
					Map.Entry<Integer, MachineEntry> MapEntry = it.next();
					// if the current machine in the Machine Database is ACTIVES
					if(MapEntry.getValue().getMachineState()==MachineState.ACTIVE)
						ClusterActiveMachineCount++;
				}
				
				SA.AddLogEntry("CMM", "Info - Learned Machine database from Training session.");
				TrainingMessageQueue = new MessageQueue();
				
				// Move to TRAINING state
				LocalMachineEntry.putMachineState(MachineState.TRAINING);
				SpreadListener.setLocalMachineState(LocalMachineEntry.getMachineState());
			}
		}
		// If local machine is in ACTIVE state or in POST_TRAINING state
		// note: TRAINING stage is handled in the listener
		else {
			// Create a new training context
			TrainingContext Context = new TrainingContext();
			Context.setLastTrainedTaskID(null);
			Context.setTraineeName(Data.getMachineName());
			Integer TrainingTaskID = null;
			try {
				// Add the new Training task into the Task database
				TrainingTaskID = InsertNewTaskIntoDB(MachineID, Context);
			} catch (CMMExn_TaskErr e) {
				// TODO remove stack trace
				e.printStackTrace();
				SA.AddLogEntry("CMM", "Error - Failed to add the new training task into the Task database.");
				SA.UnrecoverableClusterError();
				return;
			}
			SA.AddLogEntry("CMM", "Info - Added a Training task to the task database. TaskID=" + TrainingTaskID);

			// If local machine was the one who sent the message
			if(MachineID.equals(MachineDBLocalID)){
				// This training task is handled by this machine
				
				// Retrieve the Training entry for this task
				ToBeMentoredEntry FirstEntry = ToBeMentoredList.getFirst();
				if(!FirstEntry.getMachineName().equals(Data.getMachineName())) {
					// Message is not consistent with ToBeMentored Queue
					SA.AddLogEntry("CMM", "Error - Got unexpected Trainee machine name in the \"Training New Task\" message");
					SA.UnrecoverableClusterError();
					return;
				}
				
				LocalMachineEntry.incTaskCount();
				
				// Remove the training data from the ToBeMentoredList
				ToBeMentoredList.removeFirst();
				// Create a new Mentor and add it to the Mentor List
				MentorList.addLast(new Mentor(this, TrainingTaskID, Data.getMachineName(), FirstEntry.getSnapshotTaskDB(), InteractiveTrainingMode));
				
				SA.AddLogEntry("CMM", "Info - Started a new Mentor thread for this mentoring session.");
			}
		}
	}

//	------------------------------------------------------------------------	
	
	public synchronized void ReceiveTrainingUpdateTask(Integer MachineID, Integer TrainingSessionID, TaskTrainingData Data, ClusterMessage ClusterMSG) {
		
		SA.AddLogEntry("CMM", "Info - Received a \"Training Update Task\" message from the cluster." +
				" SenderID=" + MachineID + " for Trainee \"" + Data.getTraineeName() + "\", TaskID " + Data.getTrainedTaskID());
		
		// If local machine is in PRE_TRAINING state
		if(LocalMachineEntry.getMachineState() == MachineState.PRE_TRAINING) {
			// Ignore
		}
		// If local machine is in TRAINING state
		else if(LocalMachineEntry.getMachineState() == MachineState.TRAINING) {
			// If the message is addressed to this local machine
			if(LocalMachineEntry.getClusterMachineName().equals(Data.getTraineeName())){
				// Put the transfered Task entry into the Task database
				TaskDB.put(Data.getTrainedTaskID(), Data.getTrainedTaskEntry());
				
				// If this task type is PROXY
				if(Data.getTrainedTaskEntry().getContext().getType()==TaskType.PROXY){
					// Store the task's MachineID
					ProxyMachineID = Data.getTrainedTaskEntry().getAssignedServerID();
					SA.AddLogEntry("CMM", "Info - Learned that Proxy Server is currently at MachineID " + ProxyMachineID);
				}
				
				// Retrieve the assigned server of the task, and increase it's task count
				MachineDB.get(Data.getTrainedTaskEntry().getAssignedServerID()).incTaskCount();
				// Increase Cluster total task count
				ClusterTaskCount++;
				
				SA.AddLogEntry("CMM", "Info - Learned Task " + Data.getTrainedTaskID() + " from the Training session.");
			}
			// If the message is not adressed to this local machine
			else {
				// Store the message into the Message Queue to process it later
				StoreMessageWhileTraining(ClusterMSG);
			}
		}
		// If local machine is in ACTIVE state or POST_TRAINING state
		else {
			// Check if Training session exists in the Task database
			if(!TaskDB.containsKey(TrainingSessionID)) {
				SA.AddLogEntry("CMM", "Error - Training session TaskID " + TrainingSessionID + " does not exist in the Task database.");
				SA.UnrecoverableClusterError();
				return;
			}
			
			// Retrieve the training session task context from the task database
			TaskContext Context = TaskDB.get(TrainingSessionID).getContext();
			// check that the task context is of a TRAINING type
			if(Context.getType() != TaskType.TRAINING) {
				SA.AddLogEntry("CMM", "Error - Training session TaskID " + TrainingSessionID + " does not correspond with a Training task context.");
				SA.UnrecoverableClusterError();
				return;
			}
			// Check that the machine that sent this message is the assigned Mentor for this training session.
			if(!TaskDB.get(TrainingSessionID).getAssignedServerID().equals(MachineID)){
				SA.AddLogEntry("CMM", "Error - Sender MachineID (" + MachineID + ") does not match the session's Mentor MachineID " + TaskDB.get(TrainingSessionID).getAssignedServerID());
				SA.UnrecoverableClusterError();
				return;
			}
			
			
			// Update the "Last Trained TaskID" of the Training context to the TaskID of this message 
			((TrainingContext)Context).setLastTrainedTaskID(Data.getTrainedTaskID());
		}
		
	}
	
//	------------------------------------------------------------------------	
	
	public synchronized void ReceiveTrainingTerminateTask(Integer MachineID, Integer TrainingSessionID, String TraineeName, ClusterMessage ClusterMSG) {

		SA.AddLogEntry("CMM", "Info - Received a \"Training Terminate Task\" message from the cluster." +
				" SenderID=" + MachineID + " for Trainee \"" + TraineeName + "\".");
		
		// If local machine is in TRAINING
		if(LocalMachineEntry.getMachineState() == MachineState.TRAINING) {
			// If this message is addressed to this local machine
			if(LocalMachineEntry.getClusterMachineName().equals(TraineeName)){
				SA.AddLogEntry("CMM", "Info - Finished the Training session.");
				LocalMachineEntry.putMachineState(MachineState.POST_TRAINING);
				// Process all the non training messages that came 
				// during the training session 
				ProcessTaskQueue(null);
				SA.AddLogEntry("CMM", "Info - Updated the Task database according to the Training Task queue.");
				notifyAll();
			}
			// If this message is not addressed to this local machine
			else{
				// Store message in the Message Queue to proccess it later 
				StoreMessageWhileTraining(ClusterMSG);
			}
		}
		// If local machine is in ACTIVE state or in POST_TRAINING state
		else if((LocalMachineEntry.getMachineState() == MachineState.ACTIVE) ||
				(LocalMachineEntry.getMachineState() == MachineState.POST_TRAINING)){
			// Check if Training session exists in the Task database
			if(!TaskDB.containsKey(TrainingSessionID)){
				SA.AddLogEntry("CMM", "Error - Training session TaskID " + TrainingSessionID + " does not exist in the Task database.");
				SA.UnrecoverableClusterError();
				return;
			}
			// Retrieve the training session task context from the task database
			TaskContext Context = TaskDB.get(TrainingSessionID).getContext();
			// check that the task context is of a TRAINING type
			if(Context.getType() != TaskType.TRAINING) {
				SA.AddLogEntry("CMM", "Error - Training session TaskID " + TrainingSessionID + " does not correspond with a Training task context.");
				SA.UnrecoverableClusterError();
				return;
			}
			// Check that the machine that sent this message is the assigned Mentor for this training session.
			if(!TaskDB.get(TrainingSessionID).getAssignedServerID().equals(MachineID)){
				SA.AddLogEntry("CMM", "Error - Sender MachineID (" + MachineID + ") does not match the session's Mentor MachineID " + TaskDB.get(TrainingSessionID).getAssignedServerID());
				SA.UnrecoverableClusterError();
				return;
			}
			
			// Remove the Training session from the Task database
			TaskDB.remove(TrainingSessionID);
			// Decrease the task count of the ex-Mentor machine 
			MachineDB.get(MachineID).decTaskCount();
			
			if(MachineID.equals(MachineDBLocalID))
				LocalMachineEntry.decTaskCount();
			
			ClusterTaskCount--;
		}
	}
	
// -------------------------------------------------------------------------------------------------	
	
	public synchronized void ReceiveTrainingMigrate(Integer MachineID, Integer TaskID, String TraineeName, ClusterMessage_TrainingMigrate ReceivedMSG) {
		
		// Machines at PRE_TRAINING should ignore this. 
		if(LocalMachineEntry.getMachineState() == MachineState.PRE_TRAINING)
			return;
		
		SA.AddLogEntry("CMM", "Info - Received an \"Training Migrate Task\" message from MachineID="
				+ MachineID + " for TaskID=" + TaskID);
		
		// check that the MachineID is valid
		if(!MachineDB.containsKey(MachineID)){
			SA.AddLogEntry("CMM", "Error - MachineID=" + MachineID + " does not exist in the Machine Database.");
			SA.UnrecoverableClusterError();
			return;
		}
		
		if((LocalMachineEntry.getMachineState() == MachineState.ACTIVE) ||
				(LocalMachineEntry.getMachineState() == MachineState.POST_TRAINING)){
			// check that the TaskID is valid
			if(!TaskDB.containsKey(TaskID)){
				SA.AddLogEntry("CMM", "Error - TaskID=" + TaskID + " does not exist in the TaskDatabase.");
				SA.UnrecoverableClusterError();
				return;
			}
			TaskEntry MigratingTask = TaskDB.get(TaskID);
			// check that the Task is of TRAINING type
			if(MigratingTask.getContext().getType() != TaskType.TRAINING){
				SA.AddLogEntry("CMM", "Error - Task " + TaskID + " is not of TRAINING type.");
				SA.UnrecoverableClusterError();
				return;
			}
			// check that the Trainee Name is consistent with the Task Context
			if(!((TrainingContext)MigratingTask.getContext()).getTraineeName().equals(TraineeName)){
				SA.AddLogEntry("CMM", "Error - The Trainee name " + TraineeName + 
						"in the message is not consistent with the Task Database entry of: " + 
						((TrainingContext)MigratingTask.getContext()).getTraineeName());
			}
		
			// Assign the Task the new Machine
			MigratingTask.putAssignedServerID(MachineID);
			// Increase the task count of the receiving Machine
			MachineDB.get(MachineID).incTaskCount();
		
			SA.AddLogEntry("CMM", "Info - Moved Training Task " + TaskID + " to the ownership of Machine " + MachineID);
		
			// If Machine is in ACTIVE state, and message originated from this Machine
			if(MachineID.equals(MachineDBLocalID)) {
				LocalMachineEntry.incTaskCount();
				
				// This machine should start mentoring the trainee.
				SA.AddLogEntry("CMM", "Info - Training Task " + TaskID + " is being transfered to this Machine.");
			
				TreeMap<Integer, TaskEntry> SnapshotTaskDB = null;
				try {
					// Create a snapshot database from the point the prev Mentor stopped
					SnapshotTaskDB = CreateSnapshotTaskDB(((TrainingContext)TaskDB.get(TaskID).getContext()).getLastTrainedTaskID());
				} catch (CloneNotSupportedException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
					SA.AddLogEntry("CMM", "Error - Failed to create a Snapshot Task database.");
					SA.UnrecoverableClusterError();
					return;
				}
			
				// note: ToBeMentored queue is bypassed, as there is no need to wait for an 
				// additional message (not like in a new Training session) 
				// Create a new Mentor and add it to the Mentor List
				MentorList.addLast(new Mentor(this, TaskID, TraineeName, SnapshotTaskDB, InteractiveTrainingMode));
				
				SA.AddLogEntry("CMM", "Info - This Machine is the new Mentor of the Trainee " + TraineeName);
				return;
			}
		}
		
		// if this Machine is in TRAINING and the message is addressed to it
		if(LocalMachineEntry.getMachineState() == MachineState.TRAINING) {
			// if the message is addressed to the local machine
			if(TraineeName.equals(LocalMachineEntry.getClusterMachineName())) {
				// Switch Mentors
				SA.AddLogEntry("CMM", "Info - Training Task " + TaskID + " is the Training session of this Machine. Switching Mentors.");
			
				// Process the accumulated Tasks in the Task queue up to the max learned TaskID
				ProcessTaskQueue(TaskDB.lastKey());
			
				SA.AddLogEntry("CMM", "Info - Migration of the Training session to Mentor with MachineID " + MachineID + " is complete.");
				return;
			}
			// if not addressed to the local machine
			else {
				// put the message in the TaskQueue
				StoreMessageWhileTraining(ReceivedMSG);
			}
		}
		
	}

//	 -------------------------------------------------------------------------------------------------	
	
	public void StoreMessageWhileTraining(ClusterMessage message) {
		// Push the incoming message into the Training message queue 
		// for later processing.
		TrainingMessageQueue.Push(message);
	}

//	 -------------------------------------------------------------------------------------------------	
	
	public void FatalError() {
		SA.UnrecoverableClusterError();
	}
}
